/**
 * Copyright 2019-2020 Huawei Technologies Co., Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "adapter/tbe_adapter/tbe_op_store_adapter.h"
#include <thread>
#include "common/configuration.h"
#include "common/fe_error_code.h"
#include "common/fe_inner_attr_define.h"
#include "common/fe_inner_error_codes.h"
#include "common/fe_log.h"
#include "common/fe_type_utils.h"
#include "common/op_info_common.h"
#include "common/unknown_shape_util.h"
#include "ge/ge_api_types.h"
#include "graph/debug/ge_attr_define.h"
#include "graph/tuning_utils.h"
#include "graph/utils/tensor_utils.h"

namespace fe {
static const std::string TBE_SO_NAME = "libte_fusion.so";
const std::string GE_DEBUG_DIR = "ge.debugDir";
const std::string VECTOR_FP_CEILING = "ge.fpCeilingMode";
const std::string MDL_BANK_PATH_FLAG = "ge.mdl_bank_path";
const std::string OP_BANK_PATH_FLAG = "ge.op_bank_path";

static const std::map<std::string, std::string> TBE_INIT_OPTION_KEY_MAP{
    {ge::AUTO_TUNE_MODE, TBE_AUTO_TILING_MODE},
    {ge::OPTION_EXEC_DEVICE_ID, TBE_DEVICE_ID},
    {ge::SOC_VERSION, ge::SOC_VERSION},
    {ge::CORE_TYPE, ge::CORE_TYPE},
    {ge::AICORE_NUM, ge::AICORE_NUM},
    {ge::OP_SELECT_IMPL_MODE, ge::OP_SELECT_IMPL_MODE},
    {ge::OPTYPELIST_FOR_IMPLMODE, ge::OPTYPELIST_FOR_IMPLMODE},
    {ge::OP_DEBUG_LEVEL, ge::OP_DEBUG_LEVEL},
    {GE_DEBUG_DIR, GE_DEBUG_DIR},
    {ge::OP_COMPILER_CACHE_DIR, ge::OP_COMPILER_CACHE_DIR},
    {ge::OP_COMPILER_CACHE_MODE, ge::OP_COMPILER_CACHE_MODE},
    {VECTOR_FP_CEILING, VECTOR_FP_CEILING},
    {MDL_BANK_PATH_FLAG, MDL_BANK_PATH_FLAG},
    {OP_BANK_PATH_FLAG, OP_BANK_PATH_FLAG},
    {ge::PERFORMANCE_MODE, ge::PERFORMANCE_MODE}};

Status TbeOpStoreAdapter::SerialPreCompileOp(vector<PreCompileNodePara> &compile_para_vec) {
  for (auto &comp_para : compile_para_vec) {
    FE_CHECK(comp_para.node == nullptr,
             REPORT_FE_ERROR("[SubGraphOpt][Compile][SerialPreComOp] compPara.node is nullptr."),
             return FAILED);
    FE_LOGD("TbeOpStoreAdapter::PreCompile Op begin, node name: %s, node type %s.",
            comp_para.node->GetOpDesc()->GetName().c_str(), comp_para.node->GetOpDesc()->GetType().c_str());

    TbeOpInfoPtr tbe_op_info_ptr = PreCompSetTbeOpInfo(comp_para);
    if (tbe_op_info_ptr == nullptr) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][SerialPreComOp] Set TbeOpInfo Failed.");
      return FAILED;
    }

    string op_pattern_before_buff_fus;
    bool need_precompile_node = false;
    (void)ge::AttrUtils::GetBool(comp_para.node->GetOpDesc(), NEED_RE_PRECOMPILE, need_precompile_node);
    if (need_precompile_node) {
      if (!ge::AttrUtils::GetStr(comp_para.node->GetOpDesc(), comp_para.node->GetName() + "_pattern",
                                 op_pattern_before_buff_fus)) {
        FE_LOGW("Can't get BuffFus op %s pattern before precompile.", comp_para.node->GetName().c_str());
      }
    }

    FE_CHECK(PreBuildTbeOp == nullptr,
             REPORT_FE_ERROR("[SubGraphOpt][Compile][SerialPreComOp] PreBuildTbeOp is nullptr."), return FAILED);

    FE_TIMECOST_START(PreBuild);
    // call pre-compile func, and return pattern of op, such as reduction,
    bool result = PreBuildTbeOp(*tbe_op_info_ptr, 0, 0);
    if (!result) {
      std::map<std::string, std::string> error_key_map;
      // op_name,op_type,graph_id
      error_key_map[EM_OP_NAME] = comp_para.node->GetOpDesc()->GetName();
      error_key_map[EM_OP_TYPE] = comp_para.node->GetOpDesc()->GetType();
      error_key_map[EM_GRAPH_ID] = comp_para.session_graph_id;
      LogErrorMessage(EM_PRECOMPLIE_FAILED, error_key_map);
    }

    if (!result) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][SerialPreComOp] Pre-build Tbe op failed.");
      return FAILED;
    }
    FE_TIMECOST_END(PreBuild, "PreBuildTbe during FEGraphOptimizer::OptimizeFusedGraph");

    if (SetPreCompilePattern(comp_para.node->GetOpDesc(), *tbe_op_info_ptr, op_pattern_before_buff_fus) == FAILED) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][SerialPreComOp] %s set op pattern failed.",
                      comp_para.node->GetName().c_str());
      return FAILED;
    }
  }

  return SUCCESS;
}

Status TbeOpStoreAdapter::SetPreCompilePattern(ge::OpDescPtr op_desc, te::TbeOpInfo &op_info,
                                               string &op_pattern_before_buff_fus) {
  string op_pattern;
  op_info.GetPattern(op_pattern);
  if (op_pattern.empty()) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][SetPtn] %s's pattern is empty", op_desc->GetName().c_str());
    return FAILED;
  }

  FE_LOGD("op %s, pattern after precompile is %s, before precompile is %s.", op_desc->GetName().c_str(),
          op_pattern.c_str(), op_pattern_before_buff_fus.c_str());
  if (!op_pattern_before_buff_fus.empty() && (op_pattern_before_buff_fus != op_pattern)) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][SetPtn] %s's pattern (%s to %s) is changed during buffer fusion.",
                    op_desc->GetName().c_str(), op_pattern_before_buff_fus.c_str(), op_pattern.c_str());
    return FAILED;
  }

  // set op pattern to op's desc
  if (!ge::AttrUtils::SetStr(op_desc, op_desc->GetName() + "_pattern", op_pattern)) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][SetPtn] %s set pattern attr failed.", op_desc->GetName().c_str());
    return FAILED;
  }

  FE_LOGD("TbeOpStoreAdapter::PreCompile Op success. Node name: %s, node type: %s, node pattern: %s",
          op_desc->GetName().c_str(), op_desc->GetType().c_str(), op_pattern.c_str());
  return SUCCESS;
}

Status TbeOpStoreAdapter::ProcessFailPreCompTask(CompileTaskPara &task_para) {
  for (auto &fin_task : task_para.failed_tasks) {
    auto task_iter = task_para.task_node_map.find(fin_task.taskId);
    if (task_iter == task_para.task_node_map.end()) {
      REPORT_FE_ERROR("[SubGraphOpt][Pre-Comp] thread[%lu], not find task[%ld]", GetCurThreadId(), fin_task.taskId);
      return FAILED;
    }

    ge::Node *node = task_para.task_node_map[fin_task.taskId];
    REPORT_FE_ERROR("[SubGraphOpt][Pre-Comp][Node %s] Failed to pre-compile. Tid is [%lu], TaskId is [%lu] ",
                    node->GetName().c_str(), GetCurThreadId(), fin_task.taskId);
  }

  if (!task_para.failed_tasks.empty()) {
    FE_LOGD("process failed task_num[%zu]. tid[%lu]", task_para.failed_tasks.size(), GetCurThreadId());
    return FAILED;
  }

  return SUCCESS;
}

Status TbeOpStoreAdapter::ProcessSuccPreCompTask(CompileTaskPara &task_para) {
  for (auto &fin_task : task_para.succ_tasks) {
    auto task_iter = task_para.task_node_map.find(fin_task.taskId);
    if (task_iter == task_para.task_node_map.end()) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][ProSucTask] Thread[%lu], not find task[%ld]", GetCurThreadId(),
                      fin_task.taskId);
      return FAILED;
    }

    ge::Node *node = task_para.task_node_map[fin_task.taskId];
    TbeOpInfoPtr tbe_op_info_ptr = task_para.task_tbe_info_map[fin_task.taskId];
    FE_LOGD("Thread[%lu], get task[%lu], node[%s]", GetCurThreadId(), fin_task.taskId, node->GetName().c_str());

    string op_pattern_before_buff_fus;
    bool need_precompile_node = false;
    (void)ge::AttrUtils::GetBool(node->GetOpDesc(), NEED_RE_PRECOMPILE, need_precompile_node);
    if (need_precompile_node) {
      if (!ge::AttrUtils::GetStr(node->GetOpDesc(), node->GetName() + "_pattern", op_pattern_before_buff_fus)) {
        FE_LOGW("Can't get buff_fus op[%s] pattern before precompile.", node->GetName().c_str());
      }
    }

    if (SetPreCompilePattern(node->GetOpDesc(), *tbe_op_info_ptr, op_pattern_before_buff_fus) == FAILED) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][ProSucTask] %s set op pattern failed", node->GetName().c_str());
      return FAILED;
    }
  }

  FE_LOGD("process success task_num[%zu]. tid[%lu]", task_para.succ_tasks.size(), GetCurThreadId());
  return SUCCESS;
}

TbeOpInfoPtr TbeOpStoreAdapter::PreCompSetTbeOpInfo(PreCompileNodePara &comp_para) {
  if (comp_para.op_dsl_file_path.empty()) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][PreCompSetInfo] Op dsl path is invalid.");
    return nullptr;
  }

  auto op_desc = comp_para.node->GetOpDesc();
  string op_name = op_desc->GetName();
  if (!comp_para.session_graph_id.empty()) {
    op_name = comp_para.session_graph_id + "_" + op_desc->GetName();
  }

  string opFuncName = op_desc->GetType();

  TbeOpInfoPtr tbe_op_info_ptr;
  FE_MAKE_SHARED(tbe_op_info_ptr =
                     std::make_shared<te::TbeOpInfo>(op_name, comp_para.op_dsl_file_path, opFuncName, "", engine_name_),
                 return nullptr);
  tbe_op_info_ptr->SetRealName(op_desc->GetName());

  if (op_desc->HasAttr(ge::ATTR_NAME_UNREGST_OPPATH)) {
    if (tbe_single_op_info_assembler_ptr_->AssembleSingleTbeInfo(comp_para.node, *tbe_op_info_ptr, engine_name_) !=
        SUCCESS) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][PreCompSetInfo] AssembleTbeInfo failed.");
      return nullptr;
    }
  } else {
    if (tbe_info_assembler_ptr_->AssembleTbeInfo(comp_para.node, comp_para.op_kernel_info_ptr, *tbe_op_info_ptr,
                                                 engine_name_) != SUCCESS) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][PreCompSetInfo] AssembleTbeInfo failed.");
      return nullptr;
    }
  }

  if (IsFeSupportedDynamicOp(*comp_para.node->GetOpDesc())) {
    tbe_op_info_ptr->SetIsUnknownShape(true);
  }
  string op_slice_info_str;
  te::LX_QUERY_STATUS status = GetOpInfo(*tbe_op_info_ptr, op_slice_info_str);
  if (status == te::LX_QUERY_SUCC) {
    (void)ge::AttrUtils::SetStr(comp_para.node->GetOpDesc(), OP_SLICE_INFO, op_slice_info_str);
    FE_LOGD("Obtain slice info %s from tbe api for node[%s].", op_slice_info_str.c_str(),
            comp_para.node->GetName().c_str());
  } else {
    FE_LOGD("Not obtain slice info from tbe api for node[%s].", comp_para.node->GetName().c_str());
  }
  return tbe_op_info_ptr;
}
/*
 *  @ingroup fe
 *  @brief   pre-compile and return pattern of op
 *  @param   [in]  node        node pointer
 *  @param   [in]  info_store   op info store pointer
 *  @param   [in] imply_type_str  op imply type
 *  @param   [in] op_dsl_file_path  python DSL file for op
 *  @return  SUCCESS or FAILED
 */
Status TbeOpStoreAdapter::PreCompileOp(vector<PreCompileNodePara> &compile_para_vec) {
  if (!support_parallel_compile) {
    return SerialPreCompileOp(compile_para_vec);
  } else {
    return ParallelPreCompileOp(compile_para_vec);
  }
}

Status TbeOpStoreAdapter::ParallelPreCompileOp(vector<PreCompileNodePara> &compile_para_vec) {
  uint64_t thread_id = GetCurThreadId();
  CompileTaskPara task_para;
  task_para.task_num = 0;
  for (auto &comp_para : compile_para_vec) {
    FE_CHECK(comp_para.node == nullptr, REPORT_FE_ERROR("compPara.node is nullptr."), return FAILED);
    TbeOpInfoPtr tbe_op_info_ptr = PreCompSetTbeOpInfo(comp_para);
    if (tbe_op_info_ptr == nullptr) {
      REPORT_FE_ERROR("[SubGraphOpt][Pre-Comp] Set TbeOpInfo Failed.");
      return FAILED;
    }

    te::BUILD_TYPE build_type;
    bool fuzz_build = false;
    ge::AttrUtils::GetBool(comp_para.node->GetOpDesc(), ge::ATTR_NAME_FUZZ_BUILD, fuzz_build);
    if (fuzz_build) {
      build_type = te::FUZZILY_BUILD;
    } else {
      build_type = te::ACCURATELY_BUILD;
      task_para.task_num++;
    }
    tbe_op_info_ptr->SetBuildType(build_type);

    uint64_t taskId = GetAtomicId();
    task_para.task_node_map.insert(make_pair(taskId, comp_para.node));
    task_para.task_tbe_info_map.insert(make_pair(taskId, tbe_op_info_ptr));

    bool result = PreBuildTbeOp(*tbe_op_info_ptr, taskId, thread_id);
    if (!result) {
      std::map<std::string, std::string> error_key_map;
      // op_name,op_type,graph_id,thread_id,task_id
      error_key_map[EM_OP_NAME] = comp_para.node->GetOpDesc()->GetName();
      error_key_map[EM_OP_TYPE] = comp_para.node->GetOpDesc()->GetType();
      error_key_map[EM_GRAPH_ID] = comp_para.session_graph_id;
      error_key_map[EM_TASK_ID] = std::to_string(taskId);
      error_key_map[EM_THREAD_ID] = std::to_string(thread_id);

      LogErrorMessage(EM_PRECOMPLIE_TASK_FAILED, error_key_map);
    }

    if (!result) {
      REPORT_FE_ERROR("[SubGraphOpt][Pre-Comp]Failed to pre-compile node %s. thread id is [%lu], task is [%lu].",
                      comp_para.node->GetName().c_str(), thread_id, taskId);
      return FAILED;
    }
    FE_LOGD("Set precompile task[%s] success, tid[%lu], taskId[%lu].", comp_para.node->GetName().c_str(), thread_id,
            taskId);
  }

  FE_LOGD("Thread[%lu], to set %lu tasks to pre-compile.", thread_id, task_para.task_num);

  if (WaitTaskFinish(task_para) == FAILED) {
    REPORT_FE_ERROR("[SubGraphOpt][Pre-Comp]Failed to wait thread[%lu]'s task finish.", thread_id);
    return FAILED;
  }

  if (ProcessFailPreCompTask(task_para) != SUCCESS) {
    REPORT_FE_ERROR("[SubGraphOpt][Pre-Comp]Failed to process failed task. Thread_id is [%lu].", thread_id);
    return FAILED;
  }

  if (ProcessSuccPreCompTask(task_para) == FAILED) {
    REPORT_FE_ERROR("[SubGraphOpt][Pre-Comp]Failed to process successful task. Thread_id is [%lu].", thread_id);
    return FAILED;
  }

  return SUCCESS;
}

void TbeOpStoreAdapter::SetCustomFlag(ScopeNodeIdMap &fusion_nodes_map) {
  for (auto &iter : fusion_nodes_map) {
    for (ge::Node *node : iter.second) {
      if (node == nullptr) {
        continue;
      }
      string name = node->GetName();
      auto op_desc = node->GetOpDesc();
      int tmp_imply_type = 0;
      if (!ge::AttrUtils::GetInt(op_desc, FE_IMPLY_TYPE, tmp_imply_type)) {
        FE_LOGW("Node[%s]: get fe_imply_type failed.", name.c_str());
        continue;
      }

      bool is_custom_op = true;
      if (std::find(BUILT_IN_IMPLY_TYPE.begin(), BUILT_IN_IMPLY_TYPE.end(), tmp_imply_type) !=
          BUILT_IN_IMPLY_TYPE.end()) {
        is_custom_op = false;
      }

      if (!ge::AttrUtils::SetBool(op_desc, IS_CUSTOM_OP, is_custom_op)) {
        FE_LOGW("Node[%s]: set is_custom_op[%d] failed.", name.c_str(), is_custom_op);
      }
    }
  }
}

Status TbeOpStoreAdapter::CompileOp(ScopeNodeIdMap &fusion_nodes_map, map<int64_t, std::string> &json_path_map,
                                    std::vector<ge::NodePtr> &buff_fus_compile_failed_nodes,
                                    const std::vector<ge::NodePtr> &buff_fus_to_del_nodes) {
  FE_LOGD("TbeOpStoreAdapter::Compile Op begin.");
  // If the map is empty, then there is no fusion op.
  if (fusion_nodes_map.empty()) {
    FE_LOGD("Call Fusion Engine success, but there is no fusion op.");
    return SUCCESS;
  }

  SetCustomFlag(fusion_nodes_map);
  return ParallelCompileOp(fusion_nodes_map, json_path_map, buff_fus_compile_failed_nodes, buff_fus_to_del_nodes);
}

/*
 *  @ingroup fe
 *  @brief   compile fused op and single op, and generate .o and json files
 *  @param   [in]  fusion_nodes_map  op id and fused sub-graph
 *  @ptaram  [out] json_path_map    keep path of .o and json of each op
 *  @return  SUCCESS or FAILED
 */
Status TbeOpStoreAdapter::CompileOp(CompileInfoParam &compile_info) {
  FE_LOGD("TbeOpStoreAdapter::Compile Op begin.");
  // If the map is empty, then there is no fusion op.
  if (compile_info.fusion_nodes_map.empty()) {
    FE_LOGD("Call Fusion Engine success, but there is no fusion op.");
    return SUCCESS;
  }

  SetCustomFlag(compile_info.fusion_nodes_map);
  return ParallelCompileOp(compile_info.fusion_nodes_map, compile_info.scope_json_map,
                           compile_info.buff_fus_compile_failed_nodes, compile_info.buff_fus_to_del_nodes,
                           compile_info.ignore_compile_strategy, compile_info.minimum_negative_scope);
}

bool IsBuffFusOptimizedNodes(std::vector<ge::Node *> &scope_op) {
  bool need_precompile_node = false;
  Status ret_lx;
  for (auto &op : scope_op) {
    ret_lx = ge::AttrUtils::GetBool(op->GetOpDesc(), NEED_RE_PRECOMPILE, need_precompile_node);
    if (!ret_lx) {
      return false;
    }
    if (!need_precompile_node) {
      return false;
    }
  }
  return true;
}

void TbeOpStoreAdapter::SetFusionFailedId(const vector<ge::Node *> &fusion_nodes, const int64_t &fusion_failed_id) {
  for (ge::Node *node : fusion_nodes) {
    if (node == nullptr) {
      continue;
    }
    string name = node->GetName();
    if (ge::AttrUtils::SetInt(node->GetOpDesc(), FUSION_FAILED_ID_ATTR, fusion_failed_id)) {
      FE_LOGD("Node[%s]: set failed_id[%ld] success.", name.c_str(), fusion_failed_id);
    }
  }
}

bool TbeOpStoreAdapter::StopCompileOpInTuningAndAfterUBMatchMode() {
  std::string build_mode_value = Configuration::Instance(engine_name_).GetGeContextOptionValue(ge::BUILD_MODE);
  std::string step_mode_value = Configuration::Instance(engine_name_).GetGeContextOptionValue(ge::BUILD_STEP);
  if (build_mode_value == ge::BUILD_MODE_TUNING && step_mode_value == ge::BUILD_STEP_AFTER_UB_MATCH) {
    FE_LOGI("No need to try recovery if build_mode is [%s] and step is [%s].", build_mode_value.c_str(),
            step_mode_value.c_str());
    return true;
  }
  return false;
}

bool TbeOpStoreAdapter::StopWaitTaskFinishInTuningAndAfterBuilderMode(bool ignore_compile_strategy) {
  std::string build_mode_value = Configuration::Instance(engine_name_).GetGeContextOptionValue(ge::BUILD_MODE);
  std::string step_mode_value = Configuration::Instance(engine_name_).GetGeContextOptionValue(ge::BUILD_STEP);
  bool no_need_to_wait_task_finish =
      ((build_mode_value == ge::BUILD_MODE_TUNING && step_mode_value == ge::BUILD_STEP_AFTER_BUILDER) ||
       (build_mode_value == ge::BUILD_MODE_TUNING && step_mode_value == ge::BUILD_STEP_AFTER_BUILDER_SUB));
  if (!ignore_compile_strategy && no_need_to_wait_task_finish) {
    FE_LOGI("No need to wait task finish if build_mode is [%s] and step is [%s] and flag is %u.",
            build_mode_value.c_str(), step_mode_value.c_str(), ignore_compile_strategy);
    return true;
  }
  return false;
}

Status TbeOpStoreAdapter::FillInTaskParam(ScopeNodeIdMap &fusion_nodes_map, map<int64_t, std::string> &json_path_map,
                                          const std::vector<ge::NodePtr> &buff_fus_to_del_nodes,
                                          CompileTaskPara &task_para, bool ignore_compile_strategy) {
  task_para.task_num = 0;
  task_para.failed_tasks.clear();
  task_para.succ_tasks.clear();
  task_para.task_scope_id.clear();
  task_para.json_path_map = &json_path_map;
  task_para.fusion_nodes_map = &fusion_nodes_map;

  for (auto &iter : fusion_nodes_map) {
    task_para.task_num++;
    uint64_t taskId = GetAtomicId();
    task_para.task_scope_id.insert(std::make_pair(taskId, iter.first));
    FE_LOGD("%lu, taskId %lu , scope_id %ld, set compile %s task", GetCurThreadId(), taskId, iter.first,
            iter.second[0]->GetName().c_str());
    // set compile task
    if (SetTeTask(iter.second, task_para, taskId, buff_fus_to_del_nodes, ignore_compile_strategy) != SUCCESS) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][FillTaskPara] The op[%s] set compile task failed",
                      iter.second[0]->GetName().c_str());
      return FAILED;
    }
  }
  return SUCCESS;
}

Status TbeOpStoreAdapter::ProcessFailedCompileTask(CompileTaskPara &task_para,
                                                   std::vector<ge::NodePtr> &buff_fus_compile_failed_nodes,
                                                   bool ignore_compile_strategy, int64_t scope_id_minimum) {
  if (!task_para.failed_tasks.empty()) {
    std::unordered_map<uint64_t, int64_t> pre_scope_id_map = task_para.task_scope_id;
    for (auto &fin_task : task_para.failed_tasks) {
      auto task_iter = pre_scope_id_map.find(fin_task.taskId);
      if (task_iter == pre_scope_id_map.end()) {
        REPORT_FE_ERROR("[SubGraphOpt][Compile][ProcFailedCompTask] %lu, not find taskId[%ld]", GetCurThreadId(),
                        fin_task.taskId);
        return FAILED;
      }
      int64_t scope_id = pre_scope_id_map[fin_task.taskId];
      FE_LOGD("save compile %s, taskId[%lu], tid[%lu]", (*task_para.fusion_nodes_map)[scope_id][0]->GetName().c_str(),
              fin_task.taskId, GetCurThreadId());

      string node_name;
      for (auto &node : (*task_para.fusion_nodes_map)[scope_id]) {
        node_name += node->GetName();
        node_name += ", ";
      }
      FE_LOGI("Failed nodes are: {%s}", node_name.c_str());

      std::map<std::string, std::string> op_build_mapArgs = {
          {"S40000", (*task_para.fusion_nodes_map)[scope_id].at(0)->GetName()}};
      ge::ComputeGraphPtr owner_graph = (*task_para.fusion_nodes_map)[scope_id].at(0)->GetOwnerComputeGraph();
      std::string owner_graph_name;
      (void)ge::AttrUtils::GetStr(owner_graph, ge::ATTR_NAME_ROOT_GRAPH_NAME, owner_graph_name);
      SaveErrorMessage(owner_graph_name, op_build_mapArgs);
    }
    if (StopCompileOpInTuningAndAfterUBMatchMode()) {
      FE_LOGI("No need to try recovery fused op.");
      return FAILED;
    }
    if (ProcessFailCompileTask(task_para, buff_fus_compile_failed_nodes, ignore_compile_strategy, scope_id_minimum) ==
        FAILED) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][ProcFailedCompTask] Thread[%lu] process fail task failed",
                      GetCurThreadId());
      return FAILED;
    }
    // wait for finish
    if (WaitTaskFinish(task_para) == FAILED) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][ProcFailedCompTask] Thread[%lu] wait task finish failed",
                      GetCurThreadId());
      return FAILED;
    }

    if (!task_para.failed_tasks.empty()) {
      for (auto &fin_task : task_para.failed_tasks) {
        REPORT_FE_ERROR("[SubGraphOpt][Compile][ProcFailedCompTask] Thread[%lu] recompile single op[%s] failed",
                        GetCurThreadId(), fin_task.teNodeOpDesc->GetName().c_str());
      }
      return FAILED;
    }
  }
  return SUCCESS;
}

Status TbeOpStoreAdapter::ParallelCompileOp(ScopeNodeIdMap &fusion_nodes_map, map<int64_t, std::string> &json_path_map,
                                            std::vector<ge::NodePtr> &buff_fus_compile_failed_nodes,
                                            const std::vector<ge::NodePtr> &buff_fus_to_del_nodes,
                                            bool ignore_compile_strategy, int64_t scope_id_minimum) {
  FE_CHECK(TeFusion == nullptr, REPORT_FE_ERROR("[SubGraphOpt][Compile][ParalCompOp] TeFusion is nullptr."),
           return FAILED);
  FE_TIMECOST_START(TeFusion);
  CompileTaskPara task_para;
  if (FillInTaskParam(fusion_nodes_map, json_path_map, buff_fus_to_del_nodes, task_para, ignore_compile_strategy) !=
      SUCCESS) {
    return FAILED;
  }

  FE_LOGD("Thread[%lu], to set %lu tasks to comp", GetCurThreadId(), task_para.task_num);
  if (StopWaitTaskFinishInTuningAndAfterBuilderMode(ignore_compile_strategy)) {
    FE_LOGI("No need to wait task finish.");
    return SUCCESS;
  }
  // wait for finish
  if (WaitTaskFinish(task_para) == FAILED) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][ParalCompOp] Thread[%lu] wait task finish failed", GetCurThreadId());
    return FAILED;
  }
  // process success task
  if (ProcessSuccCompileTask(task_para) == FAILED) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][ParalCompOp] Thread[%lu] process success, task failed", GetCurThreadId());
    return FAILED;
  }
  // process failed task
  if (ProcessFailedCompileTask(task_para, buff_fus_compile_failed_nodes, ignore_compile_strategy, scope_id_minimum) !=
      SUCCESS) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][ParalCompOp] Thread[%lu] process fail task failed", GetCurThreadId());
    return FAILED;
  }

  // process success task
  if (ProcessSuccCompileTask(task_para) == FAILED) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][ParalCompOp] Thread[%lu] process success, task failed", GetCurThreadId());
    return FAILED;
  }

  FE_TIMECOST_END(TeFusion, "TeFusion during FEGraphOptimizer::OptimizeFusedGraph");
  FE_LOGI("TbeOpStoreAdapter::Compile Op success. tid:%lu", GetCurThreadId());

  return SUCCESS;
}

Status TbeOpStoreAdapter::SetOpJsonPath(ge::OpDescPtr &compile_op_desc, map<int64_t, std::string> &json_path_map,
                                        int scope_idx) {
  string json_file_path;
  // get json file path
  if (!(ge::AttrUtils::GetStr(compile_op_desc, "json_file_path", json_file_path))) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][SetJsonPath] Get json_file_path failed.");
    return FAILED;
  }

  if (json_file_path.empty()) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][SetJsonPath] Json path of node %s is empty.",
                    compile_op_desc->GetName().c_str());
    return FAILED;
  }
  // keep json path
  json_path_map[scope_idx] = json_file_path;
  return SUCCESS;
}

Status TbeOpStoreAdapter::DoFuzzBuildTbeOp(std::vector<ge::Node *> &node_vec, uint64_t taskId, uint64_t thread_id) {
  if (node_vec.size() == 1) {
    bool fuzz_build = false;
    ge::Node *node = node_vec[0];
    auto op_desc = node->GetOpDesc();
    (void)ge::AttrUtils::GetBool(op_desc, ge::ATTR_NAME_FUZZ_BUILD, fuzz_build);
    FE_LOGI("OP[%s] build type is %d.", node->GetName().c_str(), fuzz_build);
    if (fuzz_build) {
      bool is_support_unknown_shape = false;
      if (ge::AttrUtils::GetBool(op_desc, STR_SUPPORT_DYNAMIC_SHAPE, is_support_unknown_shape) &&
          is_support_unknown_shape) {
        // setcustomflag to node
        int tmp_imply_type = 0;
        if (!ge::AttrUtils::GetInt(op_desc, FE_IMPLY_TYPE, tmp_imply_type)) {
          FE_LOGW("Node[%s]: get fe_imply_type failed.", node->GetName().c_str());
        }
        bool is_custom_op = true;
        if (std::find(BUILT_IN_IMPLY_TYPE.begin(), BUILT_IN_IMPLY_TYPE.end(), tmp_imply_type) !=
            BUILT_IN_IMPLY_TYPE.end()) {
          is_custom_op = false;
        }
        if (!ge::AttrUtils::SetBool(op_desc, IS_CUSTOM_OP, is_custom_op)) {
          FE_LOGW("Node[%s]: set is_custom_op[%d] failed.", node->GetName().c_str(), is_custom_op);
        }

        FE_LOGD("Start to do fuzz build tbe op[%s].", node->GetName().c_str());
        te::OpBuildResCode result = FuzzBuildTbeOp(taskId, thread_id, *node);
        if (result == te::OP_BUILD_FAIL) {
          std::map<std::string, std::string> error_key_map;
          // op_name,op_type,graph_id,thread_id,task_id
          error_key_map[EM_OP_NAME] = op_desc->GetName();
          error_key_map[EM_OP_TYPE] = op_desc->GetType();
          std::string session_graph_id = "";
          (void)ge::AttrUtils::GetStr(op_desc, ge::ATTR_NAME_SESSION_GRAPH_ID, session_graph_id);
          error_key_map[EM_GRAPH_ID] = session_graph_id;
          error_key_map[EM_THREAD_ID] = std::to_string(thread_id);
          error_key_map[EM_TASK_ID] = std::to_string(taskId);
          LogErrorMessage(EM_COMPLIE_TASK_FAILED, error_key_map);
          REPORT_FE_ERROR(
              "[SubGraphOpt][Compile][DoFuzzBuild] Fuzz compile te fusion op %s failed, tid:%lu, taskId:%lu.",
              op_desc->GetName().c_str(), thread_id, taskId);
          return FAILED;
        }
        FE_LOGD("set op[%s] success, thread[%lu], taskId[%lu].", op_desc->GetName().c_str(), thread_id, taskId);
        return SUCCESS;
      }
      bool is_fuzz_build_use_accurate = false;
      bool flag_fail = !op_desc->HasAttr(STR_FUZZ_BUILD_USE_ACCURATE) ||
                       (ge::AttrUtils::GetBool(op_desc, STR_FUZZ_BUILD_USE_ACCURATE, is_fuzz_build_use_accurate) &&
                        !is_fuzz_build_use_accurate);
      if (flag_fail) {
        FE_LOGI("OP[%s] fuzz build failed.", node->GetName().c_str());
        return FAILED;
      }
    }
  }
  return NOT_CHANGED;
}

Status TbeOpStoreAdapter::SetTeTask(std::vector<ge::Node *> &node_vec, CompileTaskPara &task_para, uint64_t taskId,
                                    const std::vector<ge::NodePtr> &buff_fus_to_del_nodes,
                                    bool ignore_compile_strategy) {
  if (node_vec.empty()) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][SetTeTask] nodeVec in empty.");
    return FAILED;
  }

  std::shared_ptr<ge::OpDesc> op_desc_ptr = nullptr;
  FE_MAKE_SHARED(op_desc_ptr = std::make_shared<ge::OpDesc>(node_vec[0]->GetName(), ""), return FAILED);

  uint64_t thread_id = GetCurThreadId();

  std::string op_compile_strategy;
  (void)ge::AttrUtils::GetStr(node_vec.at(0)->GetOpDesc(), ge::ATTR_NAME_OP_COMPILE_STRATEGY, op_compile_strategy);
  FE_LOGD("Get _op_compile_strategy attr from graph is %s.", op_compile_strategy.c_str());
  FE_LOGD("Flag of ignore compile strategy is %u.", ignore_compile_strategy);
  if (ignore_compile_strategy) {
    op_compile_strategy = "set by fe: keep compiling in op tune";
  }

  // judge fuzz compile
  if (DoFuzzBuildTbeOp(node_vec, taskId, thread_id) == SUCCESS) {
    FE_LOGD("Node: %s, do fuzz build tbe op SUCCESS.", node_vec[0]->GetOpDesc()->GetName().c_str());
    return SUCCESS;
  } else if (DoFuzzBuildTbeOp(node_vec, taskId, thread_id) == FAILED) {
    REPORT_FE_ERROR("[SubGraphOpt][Compile][SetTeTask] Node: %s, do fuzz build tbe op FAILED.",
                    node_vec[0]->GetOpDesc()->GetName().c_str());
    return FAILED;
  }

  te::OpBuildResCode result =
      TeFusion(node_vec, op_desc_ptr, buff_fus_to_del_nodes, taskId, thread_id, op_compile_strategy);
  if (result == te::OP_DYNSHAPE_NOT_SUPPORT) {
    if (SetSupportDynamicShape(node_vec) != SUCCESS) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][SetTeTask] Op : %s set support_dynamicshape failed",
                      node_vec[0]->GetName().c_str());
      return FAILED;
    }
  } else if (result == te::OP_BUILD_FAIL) {
    std::map<std::string, std::string> error_key_map;

    // op_name,op_type,graph_id,thread_id,task_id
    error_key_map[EM_OP_NAME] = node_vec[0]->GetOpDesc()->GetName();
    error_key_map[EM_OP_TYPE] = node_vec[0]->GetOpDesc()->GetType();

    std::string session_graph_id = "";
    (void)ge::AttrUtils::GetStr(node_vec[0]->GetOpDesc(), ge::ATTR_NAME_SESSION_GRAPH_ID, session_graph_id);
    error_key_map[EM_GRAPH_ID] = session_graph_id;
    error_key_map[EM_THREAD_ID] = std::to_string(thread_id);
    error_key_map[EM_TASK_ID] = std::to_string(taskId);

    LogErrorMessage(EM_COMPLIE_TASK_FAILED, error_key_map);
    REPORT_FE_ERROR("[SubGraphOpt][Compile][SetTeTask] Compile te fusion op %s failed, tid:%lu, taskId:%lu.",
                    op_desc_ptr->GetName().c_str(), thread_id, taskId);
    return FAILED;
  }

  FE_LOGD("set op[%s] success, thread[%lu], taskId[%lu].", op_desc_ptr->GetName().c_str(), thread_id, taskId);
  return SUCCESS;
}

Status TbeOpStoreAdapter::WaitTaskFinish(CompileTaskPara &task_para) {
  vector<te::FinComTask> fin_com_task;
  task_para.succ_tasks.clear();
  task_para.failed_tasks.clear();

  uint64_t thread_id = GetCurThreadId();
  uint64_t task_num = task_para.task_num;
  while (task_num > 0) {
    fin_com_task.clear();
    bool ret = WaitAllFinished(thread_id, fin_com_task);
    if (!ret) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][Wait] wait for compile task finish failed. thread[%lu]", thread_id);
      return FAILED;
    }
    // not get task
    if (fin_com_task.empty()) {
      std::this_thread::sleep_for(std::chrono::milliseconds(1));
      continue;
    }

    for (auto &task : fin_com_task) {
      if (task.status == SUCCESS) {
        task_para.succ_tasks.push_back(task);
      } else {
        task_para.failed_tasks.push_back(task);
      }
      FE_LOGD("tid[%lu], taskId[%lu], task_num[%lu], fin_task_num[%lu], status[%d]", thread_id, task.taskId, task_num,
              fin_com_task.size(), task.status);
    }

    if (task_num < fin_com_task.size()) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][Wait] taskNum %lu is less than fin size %zu", task_num,
                      fin_com_task.size());
      return FAILED;
    }
    task_num -= fin_com_task.size();
  }

  FE_LOGD("tid:%lu, total_num[%lu], succ_task_num[%zu], fail_task_num[%lu]", thread_id, task_para.task_num,
          task_para.succ_tasks.size(), task_para.failed_tasks.size());
  return SUCCESS;
}

Status TbeOpStoreAdapter::ProcessSuccCompileTask(CompileTaskPara &task_para) {
  for (auto &fin_task : task_para.succ_tasks) {
    auto task_iter = task_para.task_scope_id.find(fin_task.taskId);
    if (task_iter == task_para.task_scope_id.end()) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][ProSucCmplTask] %lu, not find taskId[%ld]", GetCurThreadId(),
                      fin_task.taskId);
      return FAILED;
    }

    int64_t scope_id = task_para.task_scope_id[fin_task.taskId];
    FE_LOGD("tid[%lu], get taskId[%lu], scope_id[%ld]", GetCurThreadId(), fin_task.taskId, scope_id);
    if (SetOpJsonPath(fin_task.teNodeOpDesc, *task_para.json_path_map, scope_id) == FAILED) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][ProSucCmplTask] %s set op json path failed",
                      (*task_para.fusion_nodes_map)[scope_id][0]->GetName().c_str());
      return FAILED;
    }
    if (SetOpCompileInfo((*task_para.fusion_nodes_map)[scope_id], fin_task.teNodeOpDesc) != SUCCESS) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][ProSucCmplTask] Op : %s set op_compile_info failed",
                      (*task_para.fusion_nodes_map)[scope_id][0]->GetName().c_str());
      return FAILED;
    }
  }
  FE_LOGD("process success task_num[%zu]. tid[%lu]", task_para.succ_tasks.size(), GetCurThreadId());
  return SUCCESS;
}

void TbeOpStoreAdapter::RollBackAttributes(std::vector<ge::Node *> &failed_nodes) {
  for (auto node : failed_nodes) {
    std::vector<string> roll_back_attrs;
    auto op_desc = node->GetOpDesc();
    ge::AttrUtils::GetListStr(op_desc, ROLLBACK_IF_FAILED, roll_back_attrs);
    FE_LOGD("remove attr: node name %s size %zu", node->GetName().c_str(), roll_back_attrs.size());
    for (auto &attr : roll_back_attrs) {
      if (ge::AttrUtils::HasAttr(op_desc, attr)) {
        op_desc->DelAttr(attr);
      }
      if (attr == "reuse_input") {
        for (size_t i = 0; i < op_desc->GetAllOutputsDescSize(); i++) {
          auto out_desc = op_desc->MutableOutputDesc(i);
          if (out_desc == nullptr) {
            continue;
          }
          ge::TensorUtils::SetReuseInput(*out_desc.get(), false);
        }
        FE_LOGD("remove reuse_input for node %s.", node->GetName().c_str());
      }
    }
  }
}

Status TbeOpStoreAdapter::ProcessFailCompileTask(CompileTaskPara &task_para,
                                                 std::vector<ge::NodePtr> &buff_fus_compile_failed_nodes,
                                                 bool ignore_compile_strategy, int64_t scope_id_minimum) {
  if (task_para.failed_tasks.empty()) {
    return SUCCESS;
  }

  // Init failed id
  int64_t fusion_failed_id = GetAtomicId();
  /* In lx-fusion normal cases, the scope_idx will be started from the minimum scope id in the graph because
   * we will compile the graph twice and we will not compile those nodes which are not changed by
   * lx-fusion. */
  int scope_idx;
  if (scope_id_minimum < 0) {
    scope_idx = scope_id_minimum;
  } else {
    scope_idx = -count_if((*task_para.fusion_nodes_map).begin(), (*task_para.fusion_nodes_map).end(),
                          [](std::pair<int64_t, std::vector<ge::Node *>> item) { return item.first < 0; });
  }

  FE_LOGD("retry compilation from scope idx %d", scope_idx);
  task_para.task_num = 0;
  std::unordered_map<uint64_t, int64_t> pre_scope_id_map = task_para.task_scope_id;
  task_para.task_scope_id.clear();
  for (auto &fin_task : task_para.failed_tasks) {
    auto task_iter = pre_scope_id_map.find(fin_task.taskId);
    if (task_iter == pre_scope_id_map.end()) {
      REPORT_FE_ERROR("[SubGraphOpt][Compile][ProcFailedCompTask] tid[%lu], not find taskId[%ld]", GetCurThreadId(),
                      fin_task.taskId);
      return FAILED;
    }

    int64_t scope_id = pre_scope_id_map[fin_task.taskId];
    FE_LOGD("retry compile %s, taskId[%lu], tid[%lu]", (*task_para.fusion_nodes_map)[scope_id][0]->GetName().c_str(),
            fin_task.taskId, GetCurThreadId());

    std::vector<ge::Node *> &failed_nodes = (*task_para.fusion_nodes_map)[scope_id];
    RollBackAttributes(failed_nodes);

    if (IsBuffFusOptimizedNodes(failed_nodes)) {
      for (auto &op : failed_nodes) {
        buff_fus_compile_failed_nodes.push_back(op->shared_from_this());
        FE_LOGD("Op name = %s, type = %s.", op->GetOpDesc()->GetName().c_str(), op->GetType().c_str());
      }
      FE_LOGW("BuffFus optimized node parallel compile failed.");
      continue;
    }

    for (auto &op : failed_nodes) {
      int tmp_imply_type = 0;
      if (!ge::AttrUtils::GetInt(op->GetOpDesc(), FE_IMPLY_TYPE, tmp_imply_type)) {
        REPORT_FE_ERROR(
            "[SubGraphOpt][Compile][ProcFailedCompTask] get imply type failed, opname[%s], optype[%s],\
             op_imply_type[%d].", op->GetOpDesc()->GetName().c_str(), op->GetType().c_str(), tmp_imply_type);
        return FAILED;
      }
      if (tmp_imply_type == EN_IMPL_PLUGIN_TBE) {
        continue;
      }

      vector<ge::Node *> node_vec = {op};
      task_para.task_num++;
      scope_idx--;
      uint64_t taskId = GetAtomicId();
      task_para.task_scope_id.insert(std::make_pair(taskId, scope_idx));
      if (!(ge::AttrUtils::SetInt(op->GetOpDesc(), SCOPE_ID_ATTR, scope_idx))) {
        REPORT_FE_ERROR("[SubGraphOpt][Compile][ProcFailedCompTask] set op[%s] scope id failed.",
                        op->GetName().c_str());
        return FAILED;
      }
      FE_LOGD("Op[name:%s,type:%s] fusion failed, now single op compile, scopeid is %d.",
              op->GetOpDesc()->GetName().c_str(), op->GetType().c_str(), scope_idx);
      task_para.fusion_nodes_map->insert(make_pair(scope_idx, node_vec));
      // set compile task
      std::vector<ge::NodePtr> buff_fus_to_del_nodes;
      if (SetTeTask(node_vec, task_para, taskId, buff_fus_to_del_nodes, ignore_compile_strategy) != SUCCESS) {
        REPORT_FE_ERROR("[SubGraphOpt][Compile][ProcFailedCompTask] The op[%s] set compile task failed",
                        op->GetName().c_str());
        return FAILED;
      }
    }
    // When every op compile successful, setting failed_id for ops of fusion failed.
    std::vector<ge::Node *> fusion_failed_nodes = failed_nodes;
    SetFusionFailedId(fusion_failed_nodes, fusion_failed_id);
    fusion_failed_id++;
  }
  FE_LOGD("tid[%lu], retry task_num[%zu].", GetCurThreadId(), task_para.failed_tasks.size());
  return SUCCESS;
}

Status TbeOpStoreAdapter::InitTbeFunctions(PluginManagerPtr &plugin_manager_ptr) {
  const string TBE_SELECT_FORMAT_FUNC_NAME = "SelectTbeOpFormat";
  Status ret = plugin_manager_ptr->GetFunctionFromTbePlugin<bool, const te::TbeOpInfo &, std::string &>(
      TBE_SELECT_FORMAT_FUNC_NAME, SelectTbeOpFormat);

  if (ret != SUCCESS) {
    return FAILED;
  }

  const string TBE_CHECK_SUPPORTED_WITH_REASON_FUNC_NAME = "CheckTbeSupported";
  ret = plugin_manager_ptr->GetFunctionFromTbePlugin<bool, te::TbeOpInfo &, te::CheckSupportedResult &, std::string &>(
      TBE_CHECK_SUPPORTED_WITH_REASON_FUNC_NAME, CheckTbeSupported);
  if (ret != SUCCESS) {
    return FAILED;
  }

  const string TBE_PRE_COMPILER_FUNC_NAME = "PreBuildTbeOp";
  ret = plugin_manager_ptr->GetFunctionFromTbePlugin<bool, te::TbeOpInfo &, uint64_t, uint64_t>(
      TBE_PRE_COMPILER_FUNC_NAME, PreBuildTbeOp);
  if (ret != SUCCESS) {
    return FAILED;
  }

  const string TBE_GET_OP_INFO_FUNC_NAME = "GetOpInfo";
  ret = plugin_manager_ptr->GetFunctionFromTbePlugin<te::LX_QUERY_STATUS, const te::TbeOpInfo &, std::string &>(
      TBE_GET_OP_INFO_FUNC_NAME, GetOpInfo);
  if (ret != SUCCESS) {
    return FAILED;
  }

  const string TBE_COMPILER_FUNC_NAME = "TeFusion";
  ret = plugin_manager_ptr
            ->GetFunctionFromTbePlugin<te::OpBuildResCode, std::vector<ge::Node *>, ge::OpDescPtr,
                                       const std::vector<ge::NodePtr> &, uint64_t, uint64_t, const std::string &>(
                TBE_COMPILER_FUNC_NAME, TeFusion);
  if (ret != SUCCESS) {
    return FAILED;
  }

  const string TBE_FUZZ_COMPILER_FUNC_NAME = "FuzzBuildTbeOp";
  ret = plugin_manager_ptr->GetFunctionFromTbePlugin<te::OpBuildResCode, uint64_t, uint64_t, ge::Node &>(
      TBE_FUZZ_COMPILER_FUNC_NAME, FuzzBuildTbeOp);
  if (ret != SUCCESS) {
    return FAILED;
  }

  const string TBE_WAIT_FINISH_FUNC_NAME = "WaitAllFinished";
  ret = plugin_manager_ptr->GetFunctionFromTbePlugin<bool, uint64_t, vector<te::FinComTask> &>(
      TBE_WAIT_FINISH_FUNC_NAME, WaitAllFinished);
  if (ret != SUCCESS) {
    return FAILED;
  }

  const string TBE_INIT_FUNC_NAME = "TbeInitialize";
  ret = plugin_manager_ptr->GetFunctionFromTbePlugin<bool, const std::map<std::string, std::string> &, bool *>(
      TBE_INIT_FUNC_NAME, TbeInitialize);
  if (ret != SUCCESS) {
    return FAILED;
  }

  const string TBE_FINALIZE_FUNC_NAME = "TbeFinalize";
  ret = plugin_manager_ptr->GetFunctionFromTbePlugin<bool>(TBE_FINALIZE_FUNC_NAME, TbeFinalize);
  if (ret != SUCCESS) {
    return FAILED;
  }
  return SUCCESS;
}

Status TbeOpStoreAdapter::InitializeInnerHelp() {
  FE_MAKE_SHARED(tbe_info_assembler_ptr_ = std::make_shared<TbeInfoAssembler>(), return FAILED);
  FE_CHECK(tbe_info_assembler_ptr_ == nullptr,
           REPORT_FE_ERROR("[GraphOpt][InitializeInner][InitTbeFunc] tbeInfoAssemblerPtr_ is null."),
           return FAILED);

  FE_MAKE_SHARED(tbe_single_op_info_assembler_ptr_ = std::make_shared<TbeSingleOpInfoAssembler>(), return FAILED);
  FE_CHECK(tbe_single_op_info_assembler_ptr_ == nullptr,
           REPORT_FE_ERROR("[GraphOpt][InitializeInner][InitTbeFunc] tbeSingleOpInfoAssemblerPtr_ is null."),
           return FAILED);
  if (tbe_single_op_info_assembler_ptr_->Initialize() != SUCCESS) {
    return FAILED;
  }
  init_flag = true;
  FE_LOGI("Initialize tbe op store adapter success.");
  return SUCCESS;
}

Status TbeOpStoreAdapter::InitializeInner(const std::map<std::string, std::string> &options,
                                          const std::string &engine_name) {
  // return SUCCESS if graph optimizer has been initialized.
  if (init_flag) {
    FE_LOGW("TbeOpStoreAdapter has been initialized.");
    return SUCCESS;
  }
  /* set the engine name first */
  engine_name_ = engine_name;

  string root_path = Configuration::Instance(engine_name).GetRootPath();
  FE_LOGD("Start to initialize tbe compiler adapter.");
  const string real_path = root_path + TBE_SO_NAME;

  FE_MAKE_SHARED(plugin_manager_ptr = std::make_shared<PluginManager>(TBE_SO_NAME), return FAILED);
  FE_CHECK(plugin_manager_ptr == nullptr,
           REPORT_FE_ERROR("[GraphOpt][InitializeInner][InitTbeFunc]pluginManagerPtr is nullptr."),
           return FAILED);

  if (plugin_manager_ptr->OpenPlugin(real_path) != SUCCESS) {
    REPORT_FE_ERROR("[FEInit][OpPluginSo] Failed to open plugin so.");
    return FAILED;
  }

  Status ret = InitTbeFunctions(plugin_manager_ptr);
  if (ret != SUCCESS) {
    REPORT_FE_ERROR("[GraphOpt][InitializeInner][InitTbeFunc]: Failed to initialize TbeFunctions.");
    return FAILED;
  }

  std::map<std::string, std::string> new_options;
  for (auto key_map_iter = TBE_INIT_OPTION_KEY_MAP.begin(); key_map_iter != TBE_INIT_OPTION_KEY_MAP.end();
       key_map_iter++) {
    auto option_iter = options.find(key_map_iter->first);
    if (option_iter != options.end()) {
      new_options.insert(std::pair<string, string>(key_map_iter->second, option_iter->second));
      FE_LOGD("Options for TbeInitialize:[%s, %s]", key_map_iter->second.c_str(), option_iter->second.c_str());
    }
  }

  AppendArgsMode append_args_mode = Configuration::Instance(engine_name_).GetAppendArgsMode();
  if (append_args_mode < NO_ARGS) {
    REPORT_FE_ERROR(
        "[GraphOpt][InitializeInner][InitTbeFunc] The append_args_mode:%d is invalid,The append_args_mode \
        must greater than 0.", append_args_mode);
    return FAILED;
  }
  string append_args_mode_str = std::to_string(append_args_mode);
  new_options.insert(std::pair<string, string>(TBE_APPEND_ARGS_MODE, append_args_mode_str));

  ChangeBufferOptimize(options, new_options);
  if (!TbeInitialize(new_options, &support_parallel_compile)) {
    REPORT_FE_ERROR("[GraphOpt][InitializeInner][InitTbeFunc] Failed to init tbe.");
    if (plugin_manager_ptr->CloseHandle() != SUCCESS) {
      REPORT_FE_ERROR("[GraphOpt][InitializeInner][InitTbeFunc] Failed to close tbe plugin handle.");
    }
    return FAILED;
  }

  if (InitializeInnerHelp() != SUCCESS) {
    return FAILED;
  }
  return SUCCESS;
}
/*
 *  @ingroup fe
 *  @brief   initial resources needed by TbeOpStoreAdapter, such as dlopen so
 * files
 *           and load function symbols etc.
 *  @return  SUCCESS or FAILED
 */
Status TbeOpStoreAdapter::Initialize(const std::map<std::string, std::string> &options,
                                     const std::string &engine_name) {
  // return SUCCESS if graph optimizer has been initialized.
  Status result = InitializeInner(options, engine_name);
  if (result != SUCCESS) {
    if (plugin_manager_ptr != nullptr) {
      (void)plugin_manager_ptr->CloseHandle();
    }
    return result;
  }
  return SUCCESS;
}

void TbeOpStoreAdapter::ChangeBufferOptimize(const std::map<std::string, std::string> &options,
                                             std::map<std::string, std::string> &new_options) {
  auto iter = options.find(ge::BUFFER_OPTIMIZE);
  if (iter != options.end()) {
    if (iter->second == L2_OPTIMIZE) {
      BufferFusionMode buffer_fusion_mode = Configuration::Instance(engine_name_).GetBufferFusionMode();
      if (buffer_fusion_mode == EN_L2_FUSION) {
        new_options.insert(std::pair<string, string>(ge::BUFFER_OPTIMIZE, L2_OPTIMIZE));
      } else {
        new_options.insert(std::pair<string, string>(ge::BUFFER_OPTIMIZE, OFF_OPTIMIZE));
      }
    } else {
      new_options.insert(std::pair<string, string>(ge::BUFFER_OPTIMIZE, iter->second));
    }
  } else {
    new_options.insert(std::pair<string, string>(ge::BUFFER_OPTIMIZE, OFF_OPTIMIZE));
  }
}

/*
 *  @ingroup fe
 *  @brief   finalize resources initialized in Initialize function,
 *           such as dclose so files etc.
 *  @return  SUCCESS or FAILED
 */
Status TbeOpStoreAdapter::Finalize() {
  // return SUCCESS if graph optimizer has been initialized.
  if (!init_flag) {
    REPORT_FE_ERROR("[GraphOpt][Finalize] TbeOpStoreAdapter not allowed to finalize before initialized.");
    return FAILED;
  }

  FE_LOGD("Start to finalize tbe compiler adapter.");

  // release TBE resources
  if (!TbeFinalize()) {
    REPORT_FE_ERROR("[GraphOpt][Finalize] Release tbe resources failed.");
    return FAILED;
  }

  // close dlopen handler
  if (plugin_manager_ptr != nullptr) {
    if (plugin_manager_ptr->CloseHandle() != SUCCESS) {
      REPORT_FE_ERROR("[GraphOpt][Finalize] Failed to close tbe plugin handle.");
      return FAILED;
    }
  }

  init_flag = false;
  FE_LOGI("Finalize tbe op store adapter success.");
  return SUCCESS;
}

Status TbeOpStoreAdapter::UpdateTensorByMixPrecisionMode(ge::OpDescPtr &op_desc, OpKernelInfoPtr &op_kernel_info_ptr) {
  /* If the Auto Mix precision switch is on, we need to do the
   * checksupport in op by fp16, when the current datatype is fp32 and
   * the op is in white list or current precision mode is force fp16 */
  FE_CHECK_NOTNULL(op_kernel_info_ptr);
  bool fp16_flag = (Configuration::Instance(engine_name_).GetAutoMixPrecisionSwitch() &&
                    op_kernel_info_ptr->GetOpStoreInfo().precision_policy == WHITE) ||
                   Configuration::Instance(engine_name_).GetPrecisionModeStr() == FORCE_FP16;
  if (fp16_flag) {
    /* Set the datatype as fp16 if it's fp32 And it supports */
    for (size_t i = 0; i < op_desc->GetInputsSize(); i++) {
      auto input_desc = op_desc->GetInputDesc(i);
      if (input_desc.GetDataType() == ge::DT_FLOAT) {
        input_desc.SetDataType(ge::DT_FLOAT16);
      }
      (void)op_desc->UpdateInputDesc(i, input_desc);
    }
    for (size_t i = 0; i < op_desc->GetOutputsSize(); i++) {
      auto output_desc = op_desc->GetOutputDesc(i);
      if (output_desc.GetDataType() == ge::DT_FLOAT) {
        output_desc.SetDataType(ge::DT_FLOAT16);
      }
      (void)op_desc->UpdateOutputDesc(i, output_desc);
    }
    FE_LOGI("Node %s is in white list and the mix precision switch is on.", op_desc->GetName().c_str());
  }
  return SUCCESS;
}

/*
 *  @ingroup fe
 *  @brief   check support something
 */
bool TbeOpStoreAdapter::CheckSupport(const ge::OpDesc &op_desc, OpKernelInfoPtr op_kernel_info_ptr,
                                     std::string &reason) {
  string op_name = op_desc.GetName();
  string op_type = op_desc.GetType();

  if (!op_kernel_info_ptr->GetNeedCheckSupportFlag()) {
    return true;
  }

  FE_LOGI("[ChkSpt][OpChk][Node %s type %s] Start to check in op implementation file.", op_desc.GetName().c_str(),
          op_desc.GetType().c_str());
  /* If this op is supported in ops store, we still need to check whether
   * it is supported by specific op plugin.
   * If the mix precision switch is on, we try the dtype float16 of this op if
   * the original dtype is fp32. */
  ge::OpDescPtr op_desc_ptr_temp;
  FE_MAKE_SHARED(op_desc_ptr_temp = std::make_shared<ge::OpDesc>(op_desc),
                 REPORT_FE_ERROR("[GraphOpt][CheckSupport] Failed to create temp op_desc pointer for op[%s]",
                                 op_desc.GetName().c_str()));

  UpdateTensorByMixPrecisionMode(op_desc_ptr_temp, op_kernel_info_ptr);

  FEOpsStoreInfo op_store_info;
  if (GetTbeOpStoreInfo(op_desc, op_kernel_info_ptr, op_store_info) != SUCCESS) {
    REPORT_FE_ERROR("[GraphOpt][CheckSupport][GetTbeOpStoreInfo] Op[%s,optype[%s]]: fail to GetTbeOpStoreInfo.",
                    op_name.c_str(), op_type.c_str());
    return false;
  }

  bool is_custom_op = false;
  (void)ge::AttrUtils::GetBool(op_desc, NON_PERSISTENT_CUSTOM_OP_FLAG, is_custom_op);
  std::string op_dsl_file_path;
  bool ret_status = (is_custom_op && op_kernel_info_ptr != nullptr && !op_kernel_info_ptr->GetOpImpPath().empty());
  if (ret_status) {
    op_dsl_file_path = op_kernel_info_ptr->GetOpImpPath();
  } else {
    op_dsl_file_path = op_store_info.op_impl_file_path;
  }

  te::TbeOpInfo op_info(op_name, op_dsl_file_path, op_type, "", engine_name_);
  if (tbe_info_assembler_ptr_->AssembleTbeInfo(*op_desc_ptr_temp.get(), op_kernel_info_ptr, op_info) != SUCCESS) {
    REPORT_FE_ERROR("[GraphOpt][CheckSupport][AssembleTbeInfo] Failed to Assemble tbe info for node %s tpye %s.",
                    op_name.c_str(), op_type.c_str());
    return false;
  }

  FE_CHECK(CheckTbeSupported == nullptr,
           REPORT_FE_ERROR("[GraphOpt][CheckSupport] Function CheckTbeSupported of TeFusion is nullptr."),
           return false);

  te::CheckSupportedResult is_supported = te::NOT_SUPPORTED;
  if (CheckTbeSupported(op_info, is_supported, reason)) {
    FE_LOGI("The result of check tbe supported of op[%s] is %s.", op_desc.GetName().c_str(),
            GetCheckSupportedString(is_supported).c_str());
    bool result = ConvertCheckSupportResult(op_desc, is_supported);
    if (result) {
      FE_LOGI("[ChkSpt][OpChk][Node %s type %s] This op is supported inside its implementation.",
              op_desc.GetName().c_str(), op_desc.GetType().c_str());
    } else {
      FE_LOGI("[ChkSpt][OpChk][Node %s type %s] This op is not supported inside its implementation. Reason is %s.",
              op_desc.GetName().c_str(), op_desc.GetType().c_str(), "Null");
    }
    return result;
  }
  FE_LOGI("Fail to invoke CheckTbeSupported of TeFusion.");
  return false;
}

bool TbeOpStoreAdapter::ConvertCheckSupportResult(const ge::OpDesc &op_desc, te::CheckSupportedResult &is_supported) {
  if (is_supported == te::FULLY_SUPPORTED) {
    return true;
  } else if (is_supported == te::PARTIALLY_SUPPORTED) {
    const ge::OpDesc *tmp_op_desc = &op_desc;
    ge::OpDesc *no_const_op_desc = const_cast<ge::OpDesc *>(tmp_op_desc);
    (void)ge::AttrUtils::SetBool(*no_const_op_desc, "partially_supported", true);
    FE_LOGD("Set attr partially supported to node[%s].", op_desc.GetName().c_str());
    return true;
  } else if (is_supported == te::NOT_SUPPORTED) {
    return false;
  } else {
    return false;
  }
}

Status TbeOpStoreAdapter::SelectOpFormat(const ge::OpDesc &op_desc, const OpKernelInfoPtr &op_kernel_info_ptr,
                                         const HeavyFormatInfo &heavy_format_info, string &op_format_dtype_str) {
  string op_name = op_desc.GetName();
  string op_type = op_desc.GetType();
  FE_LOGD("Op[name=%s,type=%s]: start to SelectOpFormat.", op_name.c_str(), op_type.c_str());

  // 1. init the tbe_op_info
  FEOpsStoreInfo op_store_info;
  if (GetTbeOpStoreInfo(op_desc, op_kernel_info_ptr, op_store_info) != SUCCESS) {
    REPORT_FE_ERROR("[GraphOpt][Setcheck][SeleFormat][Op %s, type %s] fail to GetTbeOpStoreInfo.", op_name.c_str(),
                    op_type.c_str());
    return FAILED;
  }

  bool is_custom_op = false;
  (void)ge::AttrUtils::GetBool(op_desc, NON_PERSISTENT_CUSTOM_OP_FLAG, is_custom_op);
  std::string op_dsl_file_path;
  bool ret_status = (is_custom_op && op_kernel_info_ptr != nullptr && !op_kernel_info_ptr->GetOpImpPath().empty());
  if (ret_status) {
    op_dsl_file_path = op_kernel_info_ptr->GetOpImpPath();
  } else {
    op_dsl_file_path = op_store_info.op_impl_file_path;
  }

  te::TbeOpInfo tbe_op_info(op_name, op_dsl_file_path, op_type, "", engine_name_);

  // 2. assemble the information
  if (tbe_info_assembler_ptr_->AssembleTbeInfo(op_desc, op_kernel_info_ptr, heavy_format_info, tbe_op_info) !=
      SUCCESS) {
    REPORT_FE_ERROR("[GraphOpt][Setcheck][SeleFormat][Op %s, type %s] fail to assemble_tbe_info.", op_name.c_str(),
                    op_type.c_str());
    return FAILED;
  }

  // 3. call the function of TeFusion
  FE_CHECK(SelectTbeOpFormat == nullptr,
           REPORT_FE_ERROR("[GraphOpt][Setcheck][SeleFormat][Op %s, type %s] the function SelectTbeOpFormat of \
           TeFusion is nullptr.", op_name.c_str(), op_type.c_str()),
           return FAILED);
  if (!SelectTbeOpFormat(tbe_op_info, op_format_dtype_str)) {
    REPORT_CALL_ERROR(EM_SELECT_OP_FORMAT, "[OpSelectFormat][Op %s, type %s]: fail to call op select format.",
                      op_name.c_str(), op_type.c_str());
    REPORT_FE_ERROR("[GraphOpt][Setcheck][SeleFormat][Op %s, type %s] fail to call op select format.", op_name.c_str(),
                    op_type.c_str());
    return FAILED;
  }

  FE_LOGD("Op[name=%s,type=%s]: end to SelectOpFormat, op_format_dtype_str=%s.", op_desc.GetName().c_str(),
          op_desc.GetType().c_str(), op_format_dtype_str.c_str());
  return SUCCESS;
}

Status TbeOpStoreAdapter::GetTbeOpStoreInfo(const ge::OpDesc &op_desc, const OpKernelInfoPtr &op_kernel_info_ptr,
                                            FEOpsStoreInfo &op_store_info) {
  string op_name = op_desc.GetName();
  string op_type = op_desc.GetType();
  OpImplType impl_type = op_kernel_info_ptr->GetOpStoreImplType();
  if (impl_type == EN_RESERVED) {
    REPORT_FE_ERROR("[GraphOpt][Setcheck][GetOpInfo][Op %s, type %s] the imply_type [%d] is invalid.", op_name.c_str(),
                    op_type.c_str(), impl_type);
    return FAILED;
  }

  if (Configuration::Instance(engine_name_).GetOpStoreInfoByImplType(impl_type, op_store_info) != SUCCESS) {
    REPORT_FE_ERROR("[GraphOpt][Setcheck][GetOpInfo][Op %s, type %s] fail to get op store info by impl_type [%d].",
                    op_name.c_str(), op_type.c_str(), impl_type);
    return FAILED;
  }
  return SUCCESS;
}

Status TbeOpStoreAdapter::OpBuilder(ge::NodePtr node_ptr) { return SUCCESS; }

Status TbeOpStoreAdapter::SetOpCompileInfo(std::vector<ge::Node *> &nodes, const ge::OpDescPtr &op_desc_ptr) {
  for (auto &node : nodes) {
    ge::OpDescPtr cur_op_desc_ptr = node->GetOpDesc();
    FE_CHECK_NOTNULL(cur_op_desc_ptr);
    string cur_op_type = cur_op_desc_ptr->GetType();
    string cur_op_name = cur_op_desc_ptr->GetName();
    int64_t is_unknown_shape_value = 0;
    (void)ge::AttrUtils::GetInt(cur_op_desc_ptr, ATTR_NAME_IS_UNKNOWN_SHAPE_OP, is_unknown_shape_value);
    FE_LOGD("Op[name=%s,type=%s]: is_unknown_shape flag is %d.", cur_op_name.c_str(), cur_op_type.c_str(),
            is_unknown_shape_value);
    if (IsFeSupportedDynamicOp(*(cur_op_desc_ptr.get())) || is_unknown_shape_value == IS_UNKNOWN_SHAPE_VALUE) {
      std::string op_compile_info_json;
      std::string op_compile_info_key;
      if (ge::AttrUtils::GetStr(op_desc_ptr, COMPILE_INFO_JSON, op_compile_info_json)) {
        FE_LOGD("Compile info json after compiling is:%s", op_compile_info_json.c_str());
        (void)ge::AttrUtils::SetStr(cur_op_desc_ptr, COMPILE_INFO_JSON, op_compile_info_json);
      } else {
        FE_LOGW("Can not find op[name:%s,type:%s] compile_info_json after compiling.", cur_op_name.c_str(),
                cur_op_type.c_str());
      }

      if (ge::AttrUtils::GetStr(op_desc_ptr, COMPILE_INFO_KEY, op_compile_info_key)) {
        FE_LOGI("Op[name:%s,type:%s], Compile info key after compiling is:%s", cur_op_name.c_str(), cur_op_type.c_str(),
                op_compile_info_key.c_str());
        bool temp = ge::AttrUtils::SetStr(cur_op_desc_ptr, COMPILE_INFO_KEY, op_compile_info_key);
        if (temp) {
          FE_LOGI("succeed to SetStr Op[name:%s,type:%s], Compile info key after compiling is:%s", cur_op_name.c_str(),
                  cur_op_type.c_str(), op_compile_info_key.c_str());
        } else {
          FE_LOGW("Failed to Setstr Op Compile info key[name:%s,type:%s]", cur_op_name.c_str(), cur_op_type.c_str());
        }
      } else {
        FE_LOGW("Can not find op[name:%s,type:%s] compile_info_key after compiling.", cur_op_name.c_str(),
                cur_op_type.c_str());
      }
    }
  }
  return SUCCESS;
}

Status TbeOpStoreAdapter::SetSupportDynamicShape(std::vector<ge::Node *> &nodes) {
  for (auto node : nodes) {
    FE_CHECK_NOTNULL(node->GetOpDesc());
    if (IsUnKnownShapeOp(*(node->GetOpDesc().get()))) {
      (void)ge::AttrUtils::SetBool(node->GetOpDesc(), STR_SUPPORT_DYNAMIC_SHAPE, false);
    }
  }
  return SUCCESS;
}
}  // namespace fe
